
<!DOCTYPE HTML>
<html lang="zh-hans" >
    <head>
        <meta charset="UTF-8">
        <meta content="text/html; charset=utf-8" http-equiv="Content-Type">
        <title>python爬虫 · Aivin开发笔记</title>
        <meta http-equiv="X-UA-Compatible" content="IE=edge" />
        <meta name="description" content="">
        <meta name="generator" content="GitBook 3.2.3">
        <meta name="author" content="Aivin">
        
        
    
    <link rel="stylesheet" href="../gitbook/style.css">

    
            
                
                <link rel="stylesheet" href="../gitbook/gitbook-plugin-search-plus/search.css">
                
            
                
                <link rel="stylesheet" href="../gitbook/gitbook-plugin-anchor-navigation-ex/style/plugin.css">
                
            
                
                <link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css">
                
            
                
                <link rel="stylesheet" href="../gitbook/gitbook-plugin-tbfed-pagefooter/footer.css">
                
            
                
                <link rel="stylesheet" href="../gitbook/gitbook-plugin-donate/plugin.css">
                
            
                
                <link rel="stylesheet" href="../gitbook/gitbook-plugin-local-video/video-js.min.css">
                
            
                
                <link rel="stylesheet" href="../gitbook/gitbook-plugin-splitter/splitter.css">
                
            
                
                <link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css">
                
            
                
                <link rel="stylesheet" href="../gitbook/gitbook-plugin-theme-comscore/test.css">
                
            
        

    

    
        
        <link rel="stylesheet" href="../styles/website.css">
        
    

        
    
    
    
    <meta name="HandheldFriendly" content="true"/>
    <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
    <meta name="apple-mobile-web-app-capable" content="yes">
    <meta name="apple-mobile-web-app-status-bar-style" content="black">
    <link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png">
    <link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">

    
    
    <link rel="prev" href="python基础.html" />
    

    
        <link rel="shortcut icon" href='../assets/images/favicon.ico' type="image/x-icon">
    
    
        <link rel="bookmark" href='../assets/images/favicon.ico' type="image/x-icon">
    
    
    

    </head>
    <body>
        
<div class="book">
    <div class="book-summary">
        
            
<div id="book-search-input" role="search">
    <input type="text" placeholder="输入并搜索" />
</div>

            
                <nav role="navigation">
                


<ul class="summary">
    
    

    

    
        
        
    
        <li class="chapter " data-level="1.1" data-path="../">
            
                <a href="../">
            
                    
                    个人信息
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2" data-path="python基础.html">
            
                <a href="python基础.html">
            
                    
                    python基础
            
                </a>
            

            
        </li>
    
        <li class="chapter active" data-level="1.3" data-path="python爬虫.html">
            
                <a href="python爬虫.html">
            
                    
                    python爬虫
            
                </a>
            

            
        </li>
    

    

    <li class="divider"></li>

    <li>
        <a href="https://www.gitbook.com" target="blank" class="gitbook-link">
            本书使用 GitBook 发布
        </a>
    </li>
</ul>


                </nav>
            
        
    </div>

    <div class="book-body">
        
            <div class="body-inner">
                
                    

<div class="book-header" role="navigation">
    

    <!-- Title -->
    <h1>
        <i class="fa fa-circle-o-notch fa-spin"></i>
        <a href=".." >python爬虫</a>
    </h1>
</div>




                    <div class="page-wrapper" tabindex="-1" role="main">
                        <div class="page-inner">
                            
<div class="search-plus" id="book-search-results">
    <div class="search-noresults">
    
                                <section class="normal markdown-section">
                                
                                <div id="anchor-navigation-ex-navbar"><i class="fa fa-navicon"></i><ul><li><span class="title-icon "></span><a href="#&#x722C;&#x866B;"><b>1. </b>&#x722C;&#x866B;</a></li><ul><li><span class="title-icon "></span><a href="#&#x63A8;&#x8350;ide-pycharm"><b>1.1. </b>&#x63A8;&#x8350;IDE PyCharm</a></li><li><span class="title-icon "></span><a href="#pip"><b>1.2. </b>pip</a></li><ul><li><span class="title-icon "></span><a href="#pip-&#x5B89;&#x88C5;&#x5E93;"><b>1.2.1. </b>pip &#x5B89;&#x88C5;&#x5E93;</a></li></ul><li><span class="title-icon "></span><a href="#urllib&#x3001;urllib2"><b>1.3. </b>urllib&#x3001;urllib2</a></li><li><span class="title-icon "></span><a href="#beautifulsoup"><b>1.4. </b>BeautifulSoup</a></li><li><span class="title-icon "></span><a href="#&#x4E2D;&#x6587;&#x4E71;&#x7801;"><b>1.5. </b>&#x4E2D;&#x6587;&#x4E71;&#x7801;</a></li><li><span class="title-icon "></span><a href="#&#x7B80;&#x5355;demo"><b>1.6. </b>&#x7B80;&#x5355;demo</a></li><ul><li><span class="title-icon "></span><a href="#&#x6743;&#x8C0B;&#x7F51;"><b>1.6.1. </b>&#x6743;&#x8C0B;&#x7F51;</a></li><li><span class="title-icon "></span><a href="#&#x6BCF;&#x65E5;&#x6BD2;&#x9E21;&#x6C64;"><b>1.6.2. </b>&#x6BCF;&#x65E5;&#x6BD2;&#x9E21;&#x6C64;</a></li></ul></ul></ul></div><a href="#&#x722C;&#x866B;" id="anchorNavigationExGoTop"><i class="fa fa-arrow-up"></i></a><h1 id="&#x722C;&#x866B;"><a name="&#x722C;&#x866B;" class="anchor-navigation-ex-anchor" href="#&#x722C;&#x866B;"><i class="fa fa-link" aria-hidden="true"></i></a>1. &#x722C;&#x866B;</h1>
<p>Java&#x662F;&#x5F3A;&#x7C7B;&#x578B;&#xFF0C;&#x7F16;&#x8BD1;&#x578B;&#x8BED;&#x8A00;&#x3002;
Python&#x662F;&#x82E5;&#x7C7B;&#x578B;&#xFF0C;&#x89E3;&#x91CA;&#x578B;&#x8BED;&#x8A00;&#x3002;</p>
<pre><code class="lang-xml">python -V  //&#x67E5;&#x770B;&#x5F53;&#x524D;&#x7248;&#x672C;   python&#x7248;&#x672C;
import sys
print (sys.version)
</code></pre>
<h2 id="&#x63A8;&#x8350;ide-pycharm"><a name="&#x63A8;&#x8350;ide-pycharm" class="anchor-navigation-ex-anchor" href="#&#x63A8;&#x8350;ide-pycharm"><i class="fa fa-link" aria-hidden="true"></i></a>1.1. &#x63A8;&#x8350;IDE PyCharm</h2>
<h2 id="pip"><a name="pip" class="anchor-navigation-ex-anchor" href="#pip"><i class="fa fa-link" aria-hidden="true"></i></a>1.2. pip</h2>
<p>pip &#x662F;Python&#x7684;&#x8F6F;&#x4EF6;&#x5305;&#x7BA1;&#x7406;&#x7CFB;&#x7EDF;&#xFF0C;Python&#x8BED;&#x8A00;&#x81EA;&#x5E26;&#x7684;&#x547D;&#x4EE4;&#x884C;&#x5DE5;&#x5177;&#xFF0C;&#x5B83;&#x53EF;&#x4EE5;&#x5B89;&#x88C5;&#x548C;&#x7BA1;&#x7406;&#x7B2C;&#x4E09;&#x65B9;&#x8F6F;&#x4EF6;&#x5305;&#x3002;</p>
<h3 id="pip-&#x5B89;&#x88C5;&#x5E93;"><a name="pip-&#x5B89;&#x88C5;&#x5E93;" class="anchor-navigation-ex-anchor" href="#pip-&#x5B89;&#x88C5;&#x5E93;"><i class="fa fa-link" aria-hidden="true"></i></a>1.2.1. pip &#x5B89;&#x88C5;&#x5E93;</h3>
<pre><code class="lang-xml">pip install requests
http://www.lfd.uci.edu/~gohlke/pythonlibs/
</code></pre>
<h2 id="urllib&#x3001;urllib2"><a name="urllib&#x3001;urllib2" class="anchor-navigation-ex-anchor" href="#urllib&#x3001;urllib2"><i class="fa fa-link" aria-hidden="true"></i></a>1.3. urllib&#x3001;urllib2</h2>
<p>&#x5728;Python3&#x5DF2;&#x62C6;&#x5206;&#x66F4;&#x540D;&#x4E3A;urllib.request&#x548C;urllib.error</p>
<pre><code class="lang-xml">urllib  GET
f=urllib.urlopen(&quot;http://m.cnblogs.com/&quot;)
s=f.read()
print s

&#x5E26;&#x53C2;&#x6570; GET
params = urllib.urlencode({&apos;id&apos;: 8, &apos;name&apos;: &apos;jack&apos;, &apos;age&apos;: 25})
f = urllib.urlopen(&quot;http://localhost:18797/MailClient/test.aspx?%s&quot; % params)
print f.read()

POST
params = urllib.urlencode({&apos;id&apos;: 8, &apos;name&apos;: &apos;jack&apos;, &apos;age&apos;: 25})
f = urllib.urlopen(&quot;http://localhost:18797/MailClient/test.aspx&quot;,params)
print f.read()

import  urllib
URL=&apos;http://192.168.0.102:8081/findYou/pointsApi/queryAllPointsTaskFy/{offset}/{limit}&apos;
URL= URL.replace(&apos;{offset}&apos;,&apos;0&apos;)
URL = URL.replace(&apos;{limit}&apos;,&apos;1&apos;)
f=urllib.urlopen(URL)
s=f.read()
print s



//&#x4E0B;&#x8F7D;
import urllib
import urllib.request
import os
def downCallBack(a, b, c):
    &apos;&apos;&apos;&apos;&apos;
    a:&#x5DF2;&#x7ECF;&#x4E0B;&#x8F7D;&#x7684;&#x6570;&#x636E;&#x5757;
    b:&#x6570;&#x636E;&#x5757;&#x7684;&#x5927;&#x5C0F;
    c:&#x8FDC;&#x7A0B;&#x6587;&#x4EF6;&#x7684;&#x5927;&#x5C0F;
   &apos;&apos;&apos;
    per = 100.0 * a * b / c
    if per &gt; 100:
        per = 100
    print(&apos;&#x4E0B;&#x8F7D;&#x8FDB;&#x5EA6;&#xFF1A;  %.2f%%&apos; % per)

url = &apos;http://www.python.org/ftp/python/2.7.5/Python-2.7.5.tar.bz2&apos;
local = os.path.join(&apos;F:\\pythonDown\\&apos;, &apos;downDemo.zip&apos;)
urllib.request.urlretrieve(url, local, downCallBack)
</code></pre>
<h2 id="beautifulsoup"><a name="beautifulsoup" class="anchor-navigation-ex-anchor" href="#beautifulsoup"><i class="fa fa-link" aria-hidden="true"></i></a>1.4. BeautifulSoup</h2>
<pre><code class="lang-xml">pip install beautifulsoup4
</code></pre>
<h2 id="&#x4E2D;&#x6587;&#x4E71;&#x7801;"><a name="&#x4E2D;&#x6587;&#x4E71;&#x7801;" class="anchor-navigation-ex-anchor" href="#&#x4E2D;&#x6587;&#x4E71;&#x7801;"><i class="fa fa-link" aria-hidden="true"></i></a>1.5. &#x4E2D;&#x6587;&#x4E71;&#x7801;</h2>
<pre><code class="lang-xml">1&#x3001;&#x5148;&#x7528;chardet&#x68C0;&#x6D4B;&#x8BFB;&#x53D6;&#x5230;&#x7684;&#x5185;&#x5BB9;&#x7684;&#x7F16;&#x7801;
chardet.detect(content)
2&#x3001;&#x8F6C;&#x6362;&#x6210;&#x5408;&#x9002;&#x7684;&#x7F16;&#x7801;&#x683C;&#x5F0F;
html = unicode(content, &quot;gb2312&quot;).encode(&quot;utf8&quot;)
</code></pre>
<h2 id="&#x7B80;&#x5355;demo"><a name="&#x7B80;&#x5355;demo" class="anchor-navigation-ex-anchor" href="#&#x7B80;&#x5355;demo"><i class="fa fa-link" aria-hidden="true"></i></a>1.6. &#x7B80;&#x5355;demo</h2>
<h3 id="&#x6743;&#x8C0B;&#x7F51;"><a name="&#x6743;&#x8C0B;&#x7F51;" class="anchor-navigation-ex-anchor" href="#&#x6743;&#x8C0B;&#x7F51;"><i class="fa fa-link" aria-hidden="true"></i></a>1.6.1. &#x6743;&#x8C0B;&#x7F51;</h3>
<p>&#x8D44;&#x6599;&#x6765;&#x6E90; <a href="http://www.mou5.com/" target="_blank">&#x6743;&#x8C0B;&#x7F51;</a>
&#x722C;&#x866B;&#x7A0B;&#x5E8F;&#x5F88;&#x7B80;&#x5355;&#xFF0C;&#x5E76;&#x6CA1;&#x6709;&#x4F18;&#x5316;&#x3002;</p>
<h4 id="&#x83B7;&#x53D6;&#x6587;&#x7AE0;&#x5217;&#x8868;"><a name="&#x83B7;&#x53D6;&#x6587;&#x7AE0;&#x5217;&#x8868;" class="anchor-navigation-ex-anchor" href="#&#x83B7;&#x53D6;&#x6587;&#x7AE0;&#x5217;&#x8868;"><i class="fa fa-link" aria-hidden="true"></i></a>&#x83B7;&#x53D6;&#x6587;&#x7AE0;&#x5217;&#x8868;</h4>
<pre><code class="lang-python"><span class="hljs-keyword">import</span> requests
<span class="hljs-keyword">from</span> bs4 <span class="hljs-keyword">import</span> BeautifulSoup

user_agent = <span class="hljs-string">&apos;Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5)&apos;</span>
headers = {<span class="hljs-string">&apos;User-Agent&apos;</span>: user_agent}
session = requests.session()

<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">getAndSaveUrl</span><span class="hljs-params">( url)</span>:</span>
    print(url)
    page = session.get(url, headers=headers)
    soup = BeautifulSoup(page.text, <span class="hljs-string">&apos;lxml&apos;</span>)
    contents = soup.findAll(<span class="hljs-string">&apos;div&apos;</span>, id=<span class="hljs-string">&apos;contentleft&apos;</span>)
    <span class="hljs-keyword">for</span> child <span class="hljs-keyword">in</span> contents:
        item = child.findAll(<span class="hljs-string">&apos;h2&apos;</span>)
        <span class="hljs-keyword">for</span> hValue <span class="hljs-keyword">in</span> item:
            aValue = hValue.find(<span class="hljs-string">&apos;a&apos;</span>)
            title = aValue.string
            url = aValue[<span class="hljs-string">&apos;href&apos;</span>]
            print(title)
            f = open(<span class="hljs-string">&apos;D:/&#x7F51;&#x5740;.txt&apos;</span>, <span class="hljs-string">&apos;a&apos;</span>)
            f.write(<span class="hljs-string">&apos;\n&apos;</span> + url)

<span class="hljs-comment"># sum&#x5927;&#x5C0F; &#x8BF7;&#x6839;&#x636E;&#x7F51;&#x7AD9;&#x6700;&#x65B0;&#x9875;&#x6570;&#x8C03;&#x6574;</span>
sum = <span class="hljs-number">181</span>
index =<span class="hljs-number">1</span>
<span class="hljs-keyword">while</span> index &lt;= <span class="hljs-number">181</span>:
    url=<span class="hljs-string">&quot;http://www.mou5.com/page/%d&quot;</span>%(index)
    getAndSaveUrl(url)
    index = index + <span class="hljs-number">1</span>
</code></pre>
<h4 id="&#x83B7;&#x53D6;&#x4FDD;&#x5B58;&#x6587;&#x7AE0;&#x5185;&#x5BB9;"><a name="&#x83B7;&#x53D6;&#x4FDD;&#x5B58;&#x6587;&#x7AE0;&#x5185;&#x5BB9;" class="anchor-navigation-ex-anchor" href="#&#x83B7;&#x53D6;&#x4FDD;&#x5B58;&#x6587;&#x7AE0;&#x5185;&#x5BB9;"><i class="fa fa-link" aria-hidden="true"></i></a>&#x83B7;&#x53D6;&#x4FDD;&#x5B58;&#x6587;&#x7AE0;&#x5185;&#x5BB9;</h4>
<pre><code class="lang-python"><span class="hljs-keyword">import</span> random
<span class="hljs-keyword">import</span> requests
<span class="hljs-keyword">import</span> time
<span class="hljs-keyword">from</span> bs4 <span class="hljs-keyword">import</span> BeautifulSoup
<span class="hljs-keyword">import</span> re

user_agent = <span class="hljs-string">&apos;Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5)&apos;</span>
headers = {<span class="hljs-string">&apos;User-Agent&apos;</span>: user_agent}
session = requests.session()

localDir =<span class="hljs-string">&quot;C:\\test\\files\\&quot;</span>

<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">getAndSaveUrl</span><span class="hljs-params">( url)</span>:</span>
    <span class="hljs-comment"># print(&quot;&#x6293;&#x53D6; &quot;+url)</span>
    page = session.get(url, headers=headers)
    soup = BeautifulSoup(page.text, <span class="hljs-string">&apos;lxml&apos;</span>)

    contents = soup.findAll(<span class="hljs-string">&apos;div&apos;</span>, id=<span class="hljs-string">&apos;contentleft&apos;</span>)
    <span class="hljs-keyword">for</span> child <span class="hljs-keyword">in</span> contents:
        item = child.findAll(<span class="hljs-string">&apos;h2&apos;</span>)
        title= item[<span class="hljs-number">0</span>].string
        <span class="hljs-comment"># &#x53BB;&#x9664;&#x6807;&#x9898;&#x4E2D;&#x7684;&#x7279;&#x6B8A;&#x5B57;&#x7B26;</span>
        title = re.sub(<span class="hljs-string">&quot;[\s+\.\!\/_,$%^*(+\&quot;\&apos;]+|[+&#x2014;&#x2014;&#xFF01;&#xFF0C;&#x3002;?|&#xFF1F;&#x3001;~@#&#xFFE5;%&#x2026;&#x2026;&amp;*&#xFF08;&#xFF09;]+&quot;</span>, <span class="hljs-string">&quot;&quot;</span>, title)
        filePath = localDir  +title+ <span class="hljs-string">&quot;.md&quot;</span>
        f = open(filePath, <span class="hljs-string">&apos;w&apos;</span>)
        print(filePath+<span class="hljs-string">&quot;&#x521B;&#x5EFA;&#x6210;&#x529F;&quot;</span>)

        <span class="hljs-comment"># &#x4EE5;&#x4E0B;&#x4FDD;&#x5B58;&#x6B63;&#x6587;</span>
        f = open(filePath, <span class="hljs-string">&apos;a&apos;</span>, encoding=<span class="hljs-string">&apos;utf-8&apos;</span>)
        f.write(<span class="hljs-string">&apos;#  &apos;</span>)
        f.write(title+ <span class="hljs-string">&apos; \n &apos;</span>)
        f.write(<span class="hljs-string">&apos;```xml \n &apos;</span>)

        pvalues = child.findAll(<span class="hljs-string">&quot;span&quot;</span> ,attrs={<span class="hljs-string">&apos;style&apos;</span>: <span class="hljs-string">&apos;font-size:18px;&apos;</span>})
        <span class="hljs-keyword">for</span> pvalue <span class="hljs-keyword">in</span> pvalues:
            content = pvalue.string
            <span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> content <span class="hljs-keyword">is</span> <span class="hljs-keyword">None</span>:
                <span class="hljs-comment"># print(content)</span>
                f = open(filePath, <span class="hljs-string">&apos;a&apos;</span> ,encoding=<span class="hljs-string">&apos;utf-8&apos;</span>)
                f.write( content+<span class="hljs-string">&apos;\n&apos;</span> )
        f.write(<span class="hljs-string">&apos;``` \n &apos;</span>)
        f.close()
    print(<span class="hljs-string">&quot;&#x5B8C;&#x6210; &quot;</span>+url)


<span class="hljs-comment"># &#x5F00;&#x59CB;&#x8FD0;&#x884C; &#x6293;&#x53D6; &#x4FDD;&#x5B58;</span>
fileDest =localDir+<span class="hljs-string">&quot;&#x7F51;&#x5740;.txt&quot;</span>
f=open(fileDest,<span class="hljs-string">&apos;r&apos;</span>)
lines=f.readlines()
sum = len(lines)
index = <span class="hljs-number">0</span>;
<span class="hljs-keyword">for</span> line <span class="hljs-keyword">in</span> lines:
    index= index+<span class="hljs-number">1</span>
    sleepTime = random.uniform(<span class="hljs-number">0</span>, <span class="hljs-number">2</span>)
    time.sleep(sleepTime)  <span class="hljs-comment"># &#x4F11;&#x7720; 1&#x79D2;</span>
    print(<span class="hljs-string">&quot;&#x8FDB;&#x5EA6;%d/%d   ,  &#x4F11;&#x7720;%f&#x79D2;&quot;</span> % (index, sum, sleepTime))
    <span class="hljs-comment"># &#x53BB;&#x6389;&#x672B;&#x5C3E;&#x7684;&#x6362;&#x884C;&#x7B26;</span>
    url = line.strip(<span class="hljs-string">&apos;\n&apos;</span>)
    <span class="hljs-keyword">try</span>:
     getAndSaveUrl(url)
    <span class="hljs-keyword">except</span>:
        filePath = localDir  + <span class="hljs-string">&quot;errorLog.txt&quot;</span>
        f = open(filePath, <span class="hljs-string">&apos;a&apos;</span>, encoding=<span class="hljs-string">&apos;utf-8&apos;</span>)
        print(<span class="hljs-string">&quot;!!! &#x6293;&#x53D6;&#x5931;&#x8D25; &quot;</span> + url)
        f.write(url+<span class="hljs-string">&apos;\n&apos;</span>)
</code></pre>
<h4 id="&#x751F;&#x6210;-summary-&#x6587;&#x4EF6;"><a name="&#x751F;&#x6210;-summary-&#x6587;&#x4EF6;" class="anchor-navigation-ex-anchor" href="#&#x751F;&#x6210;-summary-&#x6587;&#x4EF6;"><i class="fa fa-link" aria-hidden="true"></i></a>&#x751F;&#x6210; SUMMARY &#x6587;&#x4EF6;</h4>
<pre><code class="lang-python"><span class="hljs-keyword">import</span> os.path

filePath =  <span class="hljs-string">&quot;C:\\test\\SUMMARY.md&quot;</span>
f = open(filePath, <span class="hljs-string">&apos;a&apos;</span>, encoding=<span class="hljs-string">&apos;utf-8&apos;</span>)

f.write( <span class="hljs-string">&apos;# Summary\n&apos;</span>)
f.write( <span class="hljs-string">&apos;\n&apos;</span>)
f.write( <span class="hljs-string">&apos;\n&apos;</span>)
f.write( <span class="hljs-string">&apos;* [&#x9879;&#x76EE;&#x4ECB;&#x7ECD;](README.md)&apos;</span>)
f.write( <span class="hljs-string">&apos;\n&apos;</span>)
f.write( <span class="hljs-string">&apos;* [&#x6743;&#x8C0B;&#x7F51;](chinese/&#x6743;&#x8C0B;&#x7F51;/&#x6743;&#x8C0B;&#x7F51;.md)&apos;</span>)
f.write( <span class="hljs-string">&apos;\n&apos;</span>)


rootdir =<span class="hljs-string">&quot;C:\\test\\files\\&quot;</span>

list = os.listdir(rootdir)
<span class="hljs-keyword">for</span> file <span class="hljs-keyword">in</span> list:
    fileBaseName =  os.path.basename( file)
    array = fileBaseName.split(<span class="hljs-string">&quot;.&quot;</span>)
    f.write( <span class="hljs-string">&apos;    * [&apos;</span>)
    f.write( array[<span class="hljs-number">0</span>])
    f.write(<span class="hljs-string">&apos;](chinese/&#x6743;&#x8C0B;&#x7F51;/&apos;</span>)
    f.write(fileBaseName)
    f.write( <span class="hljs-string">&apos;\n&apos;</span>)
</code></pre>
<h3 id="&#x6BCF;&#x65E5;&#x6BD2;&#x9E21;&#x6C64;"><a name="&#x6BCF;&#x65E5;&#x6BD2;&#x9E21;&#x6C64;" class="anchor-navigation-ex-anchor" href="#&#x6BCF;&#x65E5;&#x6BD2;&#x9E21;&#x6C64;"><i class="fa fa-link" aria-hidden="true"></i></a>1.6.2. &#x6BCF;&#x65E5;&#x6BD2;&#x9E21;&#x6C64;</h3>
<pre><code class="lang-java"><span class="hljs-function"><span class="hljs-keyword">public</span> <span class="hljs-keyword">static</span> <span class="hljs-keyword">void</span> <span class="hljs-title">getWeatherOfFoshanNanhai</span><span class="hljs-params">()</span> <span class="hljs-keyword">throws</span> Exception </span>{
       String url=  <span class="hljs-string">&quot;https://lwdjt.com&quot;</span>;
       String filePath =<span class="hljs-string">&quot;c://test/&#x6BD2;&#x9E21;&#x6C64;.txt&quot;</span> ;
       File file = <span class="hljs-keyword">new</span> File(filePath);
       <span class="hljs-keyword">try</span> {
           <span class="hljs-keyword">if</span>(!file.exists()){
               file.createNewFile();
           }
       } <span class="hljs-keyword">catch</span> (IOException e) {
           e.printStackTrace();
       }
       FileWriter writer = <span class="hljs-keyword">new</span> FileWriter(filePath, <span class="hljs-keyword">true</span>);

       <span class="hljs-keyword">int</span> count = <span class="hljs-number">1</span>;
       <span class="hljs-keyword">while</span> (count &lt;=<span class="hljs-number">100000</span>){
           Document doc = Jsoup.connect(url).timeout(<span class="hljs-number">2000</span>).get();
           Elements tworem = doc.select(<span class="hljs-string">&quot;main.tworem&quot;</span>);
           String str = tworem.text() ;
           <span class="hljs-comment">// &#x7E41;&#x4F53;&#x8F6C;&#x7B80;&#x4F53;</span>
           String result = ZhConverterUtil.convertToSimple(str);
           System.out.println(count+<span class="hljs-string">&quot;&#x3001;&quot;</span>+result);
           writer.write(<span class="hljs-string">&quot;\&quot;&quot;</span>);
           writer.write(result  );
           writer.write(<span class="hljs-string">&quot;\&quot;,&quot;</span>  );
           writer.write(  <span class="hljs-string">&quot;\r\n&quot;</span>);
           writer.flush();
           count++ ;
       }
       writer.close();
   }
</code></pre>
<footer class="page-footer"><span class="copyright">&#x65E0;&#x7248;&#x6743; &#x53EF;&#x4EE5;&#x65E0;&#x9650;&#x5236;&#x4F7F;&#x7528;&#xFF01; all right reserved&#xFF0C;powered by Gitbook</span><span class="footer-modification">&#x8BE5;&#x6587;&#x4EF6;&#x66F4;&#x65B0;&#x65F6;&#x95F4;&#xFF1A;
2019-07-03 11:30:36
</span></footer>
                                
                                </section>
                            
    </div>
    <div class="search-results">
        <div class="has-results">
            
            <h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
            <ul class="search-results-list"></ul>
            
        </div>
        <div class="no-results">
            
            <h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
            
        </div>
    </div>
</div>

                        </div>
                    </div>
                
            </div>

            
                
                <a href="python基础.html" class="navigation navigation-prev navigation-unique" aria-label="Previous page: python基础">
                    <i class="fa fa-angle-left"></i>
                </a>
                
                
            
        
    </div>

    <script>
        var gitbook = gitbook || [];
        gitbook.push(function() {
            gitbook.page.hasChanged({"page":{"title":"python爬虫","level":"1.3","depth":1,"previous":{"title":"python基础","level":"1.2","depth":1,"path":"chinese/python基础.md","ref":"chinese/python基础.md","articles":[]},"dir":"ltr"},"config":{"plugins":["-search","search-plus","todo","anchor-navigation-ex","copy-code-button","fontsettings","tbfed-pagefooter","donate","local-video","splitter","favicon","theme-comscore","local-video"],"styles":{"website":"styles/website.css"},"pluginsConfig":{"tbfed-pagefooter":{"copyright":"无版权 可以无限制使用！","modify_label":"该文件更新时间：","modify_format":"YYYY-MM-DD HH:mm:ss"},"todo":{},"splitter":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"donate":{"alipay":"http://oyru0ayig.bkt.clouddn.com/201803011956_osChina_支付宝收款码.png","alipayText":"支付宝打赏","button":"打赏","title":"","wechat":"http://oyru0ayig.bkt.clouddn.com/201803011956_osChina_微信收款码.png","wechatText":"微信打赏"},"fontsettings":{"family":"sans","size":2,"theme":"sepia"},"highlight":{},"anchor-navigation-ex":{"associatedWithSummary":true,"float":{"floatIcon":"fa fa-navicon","level1Icon":"","level2Icon":"","level3Icon":"","showLevelIcon":false},"mode":"float","multipleH1":true,"pageTop":{"level1Icon":"","level2Icon":"","level3Icon":"","showLevelIcon":false},"printLog":false,"showGoTop":true,"showLevel":true},"favicon":{"shortcut":"assets/images/favicon.ico","bookmark":"assets/images/favicon.ico"},"theme-comscore":{},"local-video":{},"copy-code-button":{},"sharing":{"all":["facebook","google","twitter","weibo","instapaper"],"facebook":true,"google":false,"instapaper":false,"twitter":true,"vk":false,"weibo":false},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false},"search-plus":{}},"theme":"default","author":"Aivin","pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"variables":{},"title":"Aivin开发笔记","language":"zh-hans","gitbook":"*","description":"Aivin开发笔记"},"file":{"path":"chinese/python爬虫.md","mtime":"2019-07-03T03:30:36.792Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-07-10T03:42:00.734Z"},"basePath":"..","book":{"language":""}});
        });
    </script>
</div>

        
    <script src="../gitbook/gitbook.js"></script>
    <script src="../gitbook/theme.js"></script>
    
        
        <script src="../gitbook/gitbook-plugin-search-plus/jquery.mark.min.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-search-plus/search.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-copy-code-button/toggle.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-donate/plugin.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-local-video/video.min.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-splitter/splitter.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-sharing/buttons.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-theme-comscore/test.js"></script>
        
    

    </body>
</html>

